import sys
sys.path.append('.')
import os

from util.PATH import PATH
from src.data_helper import DataHelper
import numpy as np

dataset_name = 'Eclipse'
path = PATH(dataset_name)
data_helper = DataHelper(path)
vocabulary = data_helper.create_vocabulary()
freqs = [0 for _ in range(len(vocabulary))]     # 记录每个单词出现的文档数
bug_msg_all, _ = data_helper.get_msg_all()      # In fact, 可以将这些变量一起
# 统计词汇表的词频，或者说文档数
for file in os.listdir(path.path_corpus):
	file = os.path.join(path.path_corpus, file)
	with open(file, 'r') as reader:
		# lens.append(len(reader.readlines()))
		words = set(reader.readlines())
		for word in words:
			freqs[vocabulary.index(word)] += 1
n_docs = len(os.listdir(path.path_corpus))		# 文档总数
freqs = list(map(lambda x: x/n_docs, freqs))		# 统计频率而不是频数
with open(os.path.join(path.root, 'freq.txt'), 'w') as writer:
	writer.write('\n'.join(list(map(str, freqs))))